{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-05-18T08:39:11.032647Z",
     "start_time": "2025-05-18T08:39:10.871391Z"
    }
   },
   "source": [
    "import json\n",
    "import os\n",
    "import warnings\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from mlxtend.frequent_patterns import apriori, association_rules\n",
    "from mlxtend.preprocessing import TransactionEncoder\n",
    "\n",
    "plt.rcParams['font.sans-serif'] = ['SimHei']  # 中文黑体\n",
    "plt.rcParams['axes.unicode_minus'] = False     # 正常显示负号\n",
    "\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning)"
   ],
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-18T08:39:11.037574Z",
     "start_time": "2025-05-18T08:39:11.032647Z"
    }
   },
   "cell_type": "code",
   "source": [
    "data_dir = os.path.join(os.getcwd(), \"data\")\n",
    "data_10_dir = os.path.join(data_dir, \"10G_data\")\n",
    "data_30_dir = os.path.join(data_dir, \"30G_data\")\n",
    "output_dir = os.path.join(os.getcwd(), \"output\")\n",
    "os.makedirs(output_dir, exist_ok=True)\n",
    "os.makedirs(os.path.join(output_dir, 'figures'), exist_ok=True)"
   ],
   "id": "1849b90e0eb66552",
   "outputs": [],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-18T08:39:16.693197Z",
     "start_time": "2025-05-18T08:39:11.037574Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def load_data(dir):\n",
    "    total_len, data = 0, []\n",
    "    for file in os.listdir(dir):\n",
    "        if file.endswith('.parquet'):\n",
    "            d = pd.read_parquet(os.path.join(dir, file))\n",
    "            total_len += len(d)\n",
    "            data.append(d.drop_duplicates())\n",
    "    print(f\"Loaded {total_len} rows from {os.path.basename(dir)}.\")\n",
    "    data = pd.concat(data, ignore_index=True)\n",
    "    data.drop_duplicates(inplace=True)\n",
    "    return data\n",
    "if not os.path.exists(os.path.join(data_dir, \"data_30G.parquet\")):\n",
    "    data = load_data(data_30_dir)\n",
    "    data.to_parquet(os.path.join(data_dir, \"data_30G.parquet\"), index=False)\n",
    "else:\n",
    "    data = pd.read_parquet(os.path.join(data_dir, \"data_30G.parquet\"))\n",
    "print(f\"Remain {len(data)} rows.\")"
   ],
   "id": "72763210c4b3e3cf",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remain 2400000 rows.\n"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-18T08:39:16.713518Z",
     "start_time": "2025-05-18T08:39:16.693197Z"
    }
   },
   "cell_type": "code",
   "source": "data.head()",
   "id": "4ec5246fc6a9c33b",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "   id                  timestamp   user_name chinese_name  \\\n",
       "0   1  2025-01-09T01:38:20+00:00     UZPFPZJ           彭敏   \n",
       "1   2  2023-07-08T22:53:52+00:00       UEHSG           高杰   \n",
       "2   3  2023-12-31T20:00:57+00:00    fxuujvnk          姜子轩   \n",
       "3   4  2023-03-22T11:12:02+00:00      DDERCI           梁云   \n",
       "4   5  2024-08-10T11:43:08+00:00  RTABMQKQLG           钱俊   \n",
       "\n",
       "                  email  age    income gender country  \\\n",
       "0       xtnlkqsb@qq.com   36   73000.0      女     俄罗斯   \n",
       "1  rkpktrqz@outlook.com   58  223000.0      女      巴西   \n",
       "2  vwnquvla@outlook.com   88  858000.0      男      美国   \n",
       "3       jpajekzz@qq.com   61  485000.0      男      德国   \n",
       "4      haqlhpmb@163.com   33  437000.0      男      中国   \n",
       "\n",
       "           chinese_address                                   purchase_history  \\\n",
       "0  广西壮族自治区绍兴和谐路152号2单元1384  {\"average_price\":15.940000000000001,\"category\"...   \n",
       "1     黑龙江省大连建设路127号1单元1835  {\"average_price\":563.4100000000001,\"category\":...   \n",
       "2       浙江省厦门繁荣路15号5单元1442  {\"average_price\":669.34,\"category\":\"书籍\",\"items...   \n",
       "3       四川省宁波上海路30号6单元2134  {\"average_price\":637.66,\"category\":\"书籍\",\"items...   \n",
       "4       四川省成都康乐路181号6单元391  {\"average_price\":505.0,\"category\":\"家居\",\"items\"...   \n",
       "\n",
       "   is_active registration_date  credit_score  phone_number  \n",
       "0      False        2024-10-02           423  918-668-7857  \n",
       "1      False        2021-03-19           567  205-503-3300  \n",
       "2      False        2022-05-07           767  673-105-7503  \n",
       "3      False        2020-07-08           587  387-482-7104  \n",
       "4      False        2023-10-12           404  602-478-3001  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>user_name</th>\n",
       "      <th>chinese_name</th>\n",
       "      <th>email</th>\n",
       "      <th>age</th>\n",
       "      <th>income</th>\n",
       "      <th>gender</th>\n",
       "      <th>country</th>\n",
       "      <th>chinese_address</th>\n",
       "      <th>purchase_history</th>\n",
       "      <th>is_active</th>\n",
       "      <th>registration_date</th>\n",
       "      <th>credit_score</th>\n",
       "      <th>phone_number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2025-01-09T01:38:20+00:00</td>\n",
       "      <td>UZPFPZJ</td>\n",
       "      <td>彭敏</td>\n",
       "      <td>xtnlkqsb@qq.com</td>\n",
       "      <td>36</td>\n",
       "      <td>73000.0</td>\n",
       "      <td>女</td>\n",
       "      <td>俄罗斯</td>\n",
       "      <td>广西壮族自治区绍兴和谐路152号2单元1384</td>\n",
       "      <td>{\"average_price\":15.940000000000001,\"category\"...</td>\n",
       "      <td>False</td>\n",
       "      <td>2024-10-02</td>\n",
       "      <td>423</td>\n",
       "      <td>918-668-7857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2023-07-08T22:53:52+00:00</td>\n",
       "      <td>UEHSG</td>\n",
       "      <td>高杰</td>\n",
       "      <td>rkpktrqz@outlook.com</td>\n",
       "      <td>58</td>\n",
       "      <td>223000.0</td>\n",
       "      <td>女</td>\n",
       "      <td>巴西</td>\n",
       "      <td>黑龙江省大连建设路127号1单元1835</td>\n",
       "      <td>{\"average_price\":563.4100000000001,\"category\":...</td>\n",
       "      <td>False</td>\n",
       "      <td>2021-03-19</td>\n",
       "      <td>567</td>\n",
       "      <td>205-503-3300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2023-12-31T20:00:57+00:00</td>\n",
       "      <td>fxuujvnk</td>\n",
       "      <td>姜子轩</td>\n",
       "      <td>vwnquvla@outlook.com</td>\n",
       "      <td>88</td>\n",
       "      <td>858000.0</td>\n",
       "      <td>男</td>\n",
       "      <td>美国</td>\n",
       "      <td>浙江省厦门繁荣路15号5单元1442</td>\n",
       "      <td>{\"average_price\":669.34,\"category\":\"书籍\",\"items...</td>\n",
       "      <td>False</td>\n",
       "      <td>2022-05-07</td>\n",
       "      <td>767</td>\n",
       "      <td>673-105-7503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2023-03-22T11:12:02+00:00</td>\n",
       "      <td>DDERCI</td>\n",
       "      <td>梁云</td>\n",
       "      <td>jpajekzz@qq.com</td>\n",
       "      <td>61</td>\n",
       "      <td>485000.0</td>\n",
       "      <td>男</td>\n",
       "      <td>德国</td>\n",
       "      <td>四川省宁波上海路30号6单元2134</td>\n",
       "      <td>{\"average_price\":637.66,\"category\":\"书籍\",\"items...</td>\n",
       "      <td>False</td>\n",
       "      <td>2020-07-08</td>\n",
       "      <td>587</td>\n",
       "      <td>387-482-7104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2024-08-10T11:43:08+00:00</td>\n",
       "      <td>RTABMQKQLG</td>\n",
       "      <td>钱俊</td>\n",
       "      <td>haqlhpmb@163.com</td>\n",
       "      <td>33</td>\n",
       "      <td>437000.0</td>\n",
       "      <td>男</td>\n",
       "      <td>中国</td>\n",
       "      <td>四川省成都康乐路181号6单元391</td>\n",
       "      <td>{\"average_price\":505.0,\"category\":\"家居\",\"items\"...</td>\n",
       "      <td>False</td>\n",
       "      <td>2023-10-12</td>\n",
       "      <td>404</td>\n",
       "      <td>602-478-3001</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-18T08:39:21.405826Z",
     "start_time": "2025-05-18T08:39:16.714518Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 删除无用列\n",
    "data['purchase_time'] = pd.to_datetime(data['timestamp'], utc=True)\n",
    "data.drop(columns=['id', 'chinese_name', 'email', 'age', 'chinese_address', 'country', 'timestamp', 'is_active',\n",
    "                   'gender', 'registration_date', 'credit_score', 'phone_number', 'income'], inplace=True)\n",
    "data.head()"
   ],
   "id": "3de8fb28ec264db7",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "    user_name                                   purchase_history  is_active  \\\n",
       "0     UZPFPZJ  {\"average_price\":15.940000000000001,\"category\"...      False   \n",
       "1       UEHSG  {\"average_price\":563.4100000000001,\"category\":...      False   \n",
       "2    fxuujvnk  {\"average_price\":669.34,\"category\":\"书籍\",\"items...      False   \n",
       "3      DDERCI  {\"average_price\":637.66,\"category\":\"书籍\",\"items...      False   \n",
       "4  RTABMQKQLG  {\"average_price\":505.0,\"category\":\"家居\",\"items\"...      False   \n",
       "\n",
       "              purchase_time  \n",
       "0 2025-01-09 01:38:20+00:00  \n",
       "1 2023-07-08 22:53:52+00:00  \n",
       "2 2023-12-31 20:00:57+00:00  \n",
       "3 2023-03-22 11:12:02+00:00  \n",
       "4 2024-08-10 11:43:08+00:00  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_name</th>\n",
       "      <th>purchase_history</th>\n",
       "      <th>is_active</th>\n",
       "      <th>purchase_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UZPFPZJ</td>\n",
       "      <td>{\"average_price\":15.940000000000001,\"category\"...</td>\n",
       "      <td>False</td>\n",
       "      <td>2025-01-09 01:38:20+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>UEHSG</td>\n",
       "      <td>{\"average_price\":563.4100000000001,\"category\":...</td>\n",
       "      <td>False</td>\n",
       "      <td>2023-07-08 22:53:52+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fxuujvnk</td>\n",
       "      <td>{\"average_price\":669.34,\"category\":\"书籍\",\"items...</td>\n",
       "      <td>False</td>\n",
       "      <td>2023-12-31 20:00:57+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DDERCI</td>\n",
       "      <td>{\"average_price\":637.66,\"category\":\"书籍\",\"items...</td>\n",
       "      <td>False</td>\n",
       "      <td>2023-03-22 11:12:02+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RTABMQKQLG</td>\n",
       "      <td>{\"average_price\":505.0,\"category\":\"家居\",\"items\"...</td>\n",
       "      <td>False</td>\n",
       "      <td>2024-08-10 11:43:08+00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-18T08:39:36.849213Z",
     "start_time": "2025-05-18T08:39:21.408530Z"
    }
   },
   "cell_type": "code",
   "source": [
    "product_info = json.load(open(os.path.join(data_dir, \"product_catalog.json\"), \"r\", encoding=\"utf-8\"))\n",
    "product_info = {d['id']: (d['price'], d['category']) for d in product_info['products']}\n",
    "price_map = {item_id: info[0] for item_id, info in product_info.items()}\n",
    "category_map = {item_id: info[1] for item_id, info in product_info.items()}\n",
    "print(data.purchase_history[0])\n",
    "data.purchase_history = data.purchase_history.apply(lambda x: [d[\"id\"] for d in json.loads(x)['items']])\n",
    "print(data.purchase_history[0])"
   ],
   "id": "39910d0787ee168b",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"average_price\":15.940000000000001,\"category\":\"家居\",\"items\":[{\"id\":631},{\"id\":762},{\"id\":233},{\"id\":535},{\"id\":118},{\"id\":449},{\"id\":256},{\"id\":404},{\"id\":99},{\"id\":638}]}\n",
      "[631, 762, 233, 535, 118, 449, 256, 404, 99, 638]\n"
     ]
    }
   ],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-18T08:39:36.860114Z",
     "start_time": "2025-05-18T08:39:36.849213Z"
    }
   },
   "cell_type": "code",
   "source": "data.head()",
   "id": "7c5675e40dc113ee",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "    user_name                                   purchase_history  is_active  \\\n",
       "0     UZPFPZJ  [631, 762, 233, 535, 118, 449, 256, 404, 99, 638]      False   \n",
       "1       UEHSG                                [425, 678, 72, 666]      False   \n",
       "2    fxuujvnk                               [395, 148, 731, 847]      False   \n",
       "3      DDERCI  [908, 793, 121, 530, 826, 395, 869, 510, 506, ...      False   \n",
       "4  RTABMQKQLG        [307, 439, 590, 16, 393, 738, 65, 519, 363]      False   \n",
       "\n",
       "              purchase_time  \n",
       "0 2025-01-09 01:38:20+00:00  \n",
       "1 2023-07-08 22:53:52+00:00  \n",
       "2 2023-12-31 20:00:57+00:00  \n",
       "3 2023-03-22 11:12:02+00:00  \n",
       "4 2024-08-10 11:43:08+00:00  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_name</th>\n",
       "      <th>purchase_history</th>\n",
       "      <th>is_active</th>\n",
       "      <th>purchase_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UZPFPZJ</td>\n",
       "      <td>[631, 762, 233, 535, 118, 449, 256, 404, 99, 638]</td>\n",
       "      <td>False</td>\n",
       "      <td>2025-01-09 01:38:20+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>UEHSG</td>\n",
       "      <td>[425, 678, 72, 666]</td>\n",
       "      <td>False</td>\n",
       "      <td>2023-07-08 22:53:52+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fxuujvnk</td>\n",
       "      <td>[395, 148, 731, 847]</td>\n",
       "      <td>False</td>\n",
       "      <td>2023-12-31 20:00:57+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DDERCI</td>\n",
       "      <td>[908, 793, 121, 530, 826, 395, 869, 510, 506, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>2023-03-22 11:12:02+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RTABMQKQLG</td>\n",
       "      <td>[307, 439, 590, 16, 393, 738, 65, 519, 363]</td>\n",
       "      <td>False</td>\n",
       "      <td>2024-08-10 11:43:08+00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 8
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-18T08:41:24.259285Z",
     "start_time": "2025-05-18T08:39:36.862667Z"
    }
   },
   "cell_type": "code",
   "source": [
    "t = data.copy()\n",
    "electronics_subcats = ['电子产品','智能手机','笔记本电脑','平板电脑','智能手表', '耳机','音响','相机','摄像机','游戏机']\n",
    "t['categories'] = t['purchase_history'].apply(lambda ids: [category_map[i] if category_map[i] not in electronics_subcats else '电子产品' for i in ids])\n",
    "\n",
    "# --- Task 1: Category Association Rules ---\n",
    "te = TransactionEncoder()\n",
    "te_ary = te.fit(t['categories']).transform(t['categories'])\n",
    "basket_df = pd.DataFrame(te_ary, columns=te.columns_)\n",
    "\n",
    "freq_itemsets_all = apriori(basket_df, min_support=0.02, use_colnames=True)\n",
    "rules_all = association_rules(freq_itemsets_all, metric=\"confidence\", min_threshold=0.5)\n",
    "print(\"Task 1a: All Categories Association Rules (support>=0.02, confidence>=0.5):\")\n",
    "print(rules_all[['antecedents','consequents','support','confidence','lift']]\n",
    "      .sort_values(by=['lift','support'], ascending=[False,False]).head(10))\n",
    "\n",
    "# Focus on '电子产品' associations\n",
    "rules_elec = rules_all[rules_all['antecedents'].apply(lambda x: '电子产品' in x) |\n",
    "                        rules_all['consequents'].apply(lambda x: '电子产品' in x)]\n",
    "print(\"\\nTask 1b: Association Rules Involving '电子产品':\")\n",
    "print(rules_elec[['antecedents','consequents','support','confidence','lift']]\n",
    "      .sort_values(by=['lift','support'], ascending=[False,False]).head(10))\n",
    "# Export Task 1 results\n",
    "rules_all.to_csv(os.path.join(output_dir, 'all_rules.csv'), index=False)\n",
    "rules_elec.to_csv(os.path.join(output_dir, 'electronics_rules.csv'), index=False)\n",
    "print(\"Task 1 results exported to CSV: all_rules.csv & electronics_rules.csv\")"
   ],
   "id": "d45267d63075cdb",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task 1a: All Categories Association Rules (support>=0.02, confidence>=0.5):\n",
      "     antecedents consequents   support  confidence      lift\n",
      "36    (户外装备, 蔬菜)      (电子产品)  0.021131    0.716441  1.110918\n",
      "39      (蔬菜, 饮料)      (电子产品)  0.022961    0.716373  1.110812\n",
      "34    (卫浴用品, 蔬菜)      (电子产品)  0.021878    0.716115  1.110413\n",
      "35    (卫浴用品, 饮料)      (电子产品)  0.022257    0.714785  1.108349\n",
      "33  (户外装备, 卫浴用品)      (电子产品)  0.020563    0.714366  1.107701\n",
      "37    (户外装备, 饮料)      (电子产品)  0.021753    0.713269  1.105999\n",
      "40      (饮料, 零食)      (电子产品)  0.020156    0.712172  1.104298\n",
      "38      (文具, 饮料)      (电子产品)  0.020340    0.711527  1.103298\n",
      "19        (汽车装饰)      (电子产品)  0.047793    0.694406  1.076749\n",
      "28         (调味品)      (电子产品)  0.065140    0.693852  1.075891\n",
      "\n",
      "Task 1b: Association Rules Involving '电子产品':\n",
      "     antecedents consequents   support  confidence      lift\n",
      "36    (户外装备, 蔬菜)      (电子产品)  0.021131    0.716441  1.110918\n",
      "39      (蔬菜, 饮料)      (电子产品)  0.022961    0.716373  1.110812\n",
      "34    (卫浴用品, 蔬菜)      (电子产品)  0.021878    0.716115  1.110413\n",
      "35    (卫浴用品, 饮料)      (电子产品)  0.022257    0.714785  1.108349\n",
      "33  (户外装备, 卫浴用品)      (电子产品)  0.020563    0.714366  1.107701\n",
      "37    (户外装备, 饮料)      (电子产品)  0.021753    0.713269  1.105999\n",
      "40      (饮料, 零食)      (电子产品)  0.020156    0.712172  1.104298\n",
      "38      (文具, 饮料)      (电子产品)  0.020340    0.711527  1.103298\n",
      "19        (汽车装饰)      (电子产品)  0.047793    0.694406  1.076749\n",
      "28         (调味品)      (电子产品)  0.065140    0.693852  1.075891\n",
      "Task 1 results exported to CSV: all_rules.csv & electronics_rules.csv\n"
     ]
    }
   ],
   "execution_count": 9
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-18T08:41:24.286615Z",
     "start_time": "2025-05-18T08:41:24.267301Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# # --- Task 2: Payment Method vs Category ---\n",
    "# pay_df = t.explode('categories')[['payment_method','categories']]\n",
    "# pay_dummy = pd.get_dummies(pay_df)\n",
    "# pay_group = pay_dummy.groupby(level=0).sum()\n",
    "# \n",
    "# freq_pay = apriori(pay_group, min_support=0.01, use_colnames=True)\n",
    "# rules_pay = association_rules(freq_pay, metric='confidence', min_threshold=0.6)\n",
    "# print(\"\\nTask 2a: Payment-Category rules (top10 by confidence):\")\n",
    "# print(rules_pay[['antecedents','consequents','support','confidence','lift']]\\\n",
    "#       .sort_values('confidence', ascending=False).head(10))\n",
    "# \n",
    "# # High-value (>5000) items\n",
    "# t['high_value'] = t['purchase_history'].apply(lambda ids: any(product_info[i][0] > 5000 for i in ids))\n",
    "# hv = t[t['high_value']]\n",
    "# hv_counts = hv['payment_method'].value_counts()\n",
    "# print(\"\\nTask 2b: High-value item payment method distribution:\")\n",
    "# print(hv_counts)\n",
    "# \n",
    "# # Export Task2\n",
    "# rules_pay.to_csv(os.path.join(output_dir, 'payment_rules.csv'), index=False)\n",
    "# hv_counts.to_csv(os.path.join(output_dir, 'high_value_payment_counts.csv'), header=['count'])"
   ],
   "id": "8cfdcd0fdb42a3f8",
   "outputs": [],
   "execution_count": 10
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-18T08:42:18.002824Z",
     "start_time": "2025-05-18T08:41:24.294496Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# --- Task 3: Time Series Patterns ---\n",
    "ts = t.set_index('purchase_time').explode('categories')\n",
    "\n",
    "monthly_stats = {}\n",
    "for cat in ts['categories'].unique():\n",
    "    monthly_freq = ts[ts['categories']==cat].resample('M').size()\n",
    "    monthly_stats[cat] = monthly_freq\n",
    "    # Plot each\n",
    "    plt.figure()\n",
    "    plt.plot(monthly_freq)\n",
    "    plt.title(f\"Monthly Purchase Frequency: {cat}\")\n",
    "    plt.xlabel('Month')\n",
    "    plt.ylabel('Count')\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(os.path.join(output_dir, 'figures', f\"time_series_{cat}.png\"))\n",
    "    plt.close()\n",
    "\n",
    "print(\"\\nTask 3: Time series plots saved for each category.\")\n",
    "# Optionally, combine monthly_stats into DataFrame and export\n",
    "df_monthly = pd.DataFrame(monthly_stats).fillna(0)\n",
    "df_monthly.to_csv(os.path.join(output_dir, 'monthly_category_freq.csv'))"
   ],
   "id": "ddeb66f59f32d000",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Task 3: Time series plots saved for each category.\n"
     ]
    }
   ],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-18T08:42:29.751620Z",
     "start_time": "2025-05-18T08:42:18.002824Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# --- Task 4: Refund Pattern Analysis ---\n",
    "refunds = t[t['is_active']==False]['categories'].tolist()\n",
    "te_ref = TransactionEncoder()\n",
    "te_ref_ary = te_ref.fit(refunds).transform(refunds)\n",
    "ref_df = pd.DataFrame(te_ref_ary, columns=te_ref.columns_)\n",
    "\n",
    "freq_ref = apriori(ref_df, min_support=0.005, use_colnames=True)\n",
    "rules_ref = association_rules(freq_ref, metric='confidence', min_threshold=0.4)\n",
    "print(\"\\nTask 4: Refund-related rules (top10 by support):\")\n",
    "print(rules_ref[['antecedents','consequents','support','confidence','lift']]\\\n",
    "      .sort_values('support', ascending=False).head(10))\n",
    "\n",
    "# Export Task4\n",
    "rules_ref.to_csv('refund_rules.csv', index=False)\n",
    "print(\"\\nAll tasks completed. Results and plots exported.\")"
   ],
   "id": "e482b911bdebb29c",
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "Unable to allocate 40.1 GiB for an array with shape (5984, 3, 2400000) and data type bool",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mMemoryError\u001B[0m                               Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[12], line 7\u001B[0m\n\u001B[0;32m      4\u001B[0m te_ref_ary \u001B[38;5;241m=\u001B[39m te_ref\u001B[38;5;241m.\u001B[39mfit(refunds)\u001B[38;5;241m.\u001B[39mtransform(refunds)\n\u001B[0;32m      5\u001B[0m ref_df \u001B[38;5;241m=\u001B[39m pd\u001B[38;5;241m.\u001B[39mDataFrame(te_ref_ary, columns\u001B[38;5;241m=\u001B[39mte_ref\u001B[38;5;241m.\u001B[39mcolumns_)\n\u001B[1;32m----> 7\u001B[0m freq_ref \u001B[38;5;241m=\u001B[39m apriori(ref_df, min_support\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m0.005\u001B[39m, use_colnames\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m)\n\u001B[0;32m      8\u001B[0m rules_ref \u001B[38;5;241m=\u001B[39m association_rules(freq_ref, metric\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mconfidence\u001B[39m\u001B[38;5;124m'\u001B[39m, min_threshold\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m0.4\u001B[39m)\n\u001B[0;32m      9\u001B[0m \u001B[38;5;28mprint\u001B[39m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;130;01m\\n\u001B[39;00m\u001B[38;5;124mTask 4: Refund-related rules (top10 by support):\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n",
      "File \u001B[1;32mD:\\ProgramData\\minicoda3\\Lib\\site-packages\\mlxtend\\frequent_patterns\\apriori.py:309\u001B[0m, in \u001B[0;36mapriori\u001B[1;34m(df, min_support, use_colnames, max_len, verbose, low_memory)\u001B[0m\n\u001B[0;32m    307\u001B[0m         _bools \u001B[38;5;241m=\u001B[39m _bools \u001B[38;5;241m&\u001B[39m (X[:, combin[:, n]] \u001B[38;5;241m==\u001B[39m all_ones)\n\u001B[0;32m    308\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m--> 309\u001B[0m     _bools \u001B[38;5;241m=\u001B[39m np\u001B[38;5;241m.\u001B[39mall(X[:, combin], axis\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m2\u001B[39m)\n\u001B[0;32m    311\u001B[0m support \u001B[38;5;241m=\u001B[39m _support(np\u001B[38;5;241m.\u001B[39marray(_bools), rows_count, is_sparse)\n\u001B[0;32m    312\u001B[0m _mask \u001B[38;5;241m=\u001B[39m (support \u001B[38;5;241m>\u001B[39m\u001B[38;5;241m=\u001B[39m min_support)\u001B[38;5;241m.\u001B[39mreshape(\u001B[38;5;241m-\u001B[39m\u001B[38;5;241m1\u001B[39m)\n",
      "\u001B[1;31mMemoryError\u001B[0m: Unable to allocate 40.1 GiB for an array with shape (5984, 3, 2400000) and data type bool"
     ]
    }
   ],
   "execution_count": 12
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
